const rp = require('request-promise');
const cheerio = require('cheerio');
const iconv = require('iconv-lite');
const moment = require('moment');
const mkdirp = require('mkdirp');
const fs = require('fs');
const imgHost = '';
let isGreping = false;
// http://m3.clz99.com
const imgSaveDir = '/www/nodepro/homelive/imgsite/www';

module.exports = class extends think.Controller {
  // 数据采集
  async dataCollectAction() {
    if (isGreping) {
      return this.fail('it is greping');
    }
    isGreping = true;
    const baseUrl = 'http://www.27270.com/word/';
    const content = await this.fetchLinkContent(baseUrl);
    const $ = cheerio.load(content, {decodeEntities: false});
    this.loadforcatetory(1, $);
  }
  async loadforcatetory(id, $) {
    const aNodes = $('#masonryList').find('>li>a');
    const albums = [];
    for (let i = 0, len = aNodes.length; i < len; i++) {
      const node = $(aNodes[i]);
      const link = node.attr('href') || '';
      let cover = node.find('img').attr('src');
      if (cover && link) {
        if (cover.endsWith('defaultpic.gif')) {
          cover = '';
        }
        try {
          const content = await this.fetchAlbumContent(link);
          if (think.isEmpty(content)) {
            continue;
          }
          if (cover) {
            // 保存封面
            const url = this.generateImgName(cover);
            this.downloadImg(cover, url);
            cover = url;
          }
          albums.push({
            sourceUrl: link,
            cover,
            ...content
          });
        } catch (e) {
          think.logger.info('fetch link:failed:', link);
        }
        // break;
      }
    }
    this.saveAlbums(id, albums);
  }
  // 数据库存储
  async saveAlbums(categoryId, albums) {
    const albumModel = think.model('album');
    const imgModel = think.model('img').db(albumModel.db());
    albums.forEach(async album => {
      albumModel.transaction(async() => {
        const albumId = await albumModel.add({
          cover: album.cover,
          type_id: album.type,
          source_url: album.sourceUrl,
          content: album.content,
          create_time: think.datetime(),
          title: album.title,
          category_id: categoryId
        });
        if (album.imgSrcs && album.imgSrcs.length) {
          await imgModel.addMany(album.imgSrcs.map(img => {
            return {
              img_src: img,
              album_id: albumId,
              create_time: think.datetime()
            };
          }));
          think.logger.info('save album success:', album.title);
        }
      });
    });
  }
  async fetchLinkContent(link) {
    think.logger.info('fetch link:', link);
    let content = await rp.get({
      encoding: null,
      url: link
    });
    content = iconv.decode(content, 'gb2312').toString();
    return content;
  }
  async albumExistsByTitle(title) {
    const albumModel = think.model('album');
    return await albumModel.where({
      title
    }).count() > 0;
  }
  async fetchAlbumContent(link) {
    // link = 'http://www.27270.com/beautiful/zhuomianbeijing/2018/307931.html';
    const content = await this.fetchLinkContent(link);
    const $ = cheerio.load(content, {decodeEntities: false});
    const body = $('.contentV3_body');
    if (body.length === 0) { // is album pictures
      return this.loadPicAlbum($, link);
    } else { // is news
      return this.loadNews($);
    }
  }
  async loadPicAlbum($, link) {
    const title = $('.articleV4Tit').text().trim();
    if (think.isEmpty(title) && await this.albumExistsByTitle(title)) {
      return;
    }
    const content = $('.articleV4Desc > p').text().trim() || '';
    const imgSrc = $('#picBody').find('img').attr('src');
    if (think.isEmpty(imgSrc)) {
      return;
    }
    const keywords = ($('meta[name=keywords]').attr('content') || '').trim();
    const description = ($('meta[name=description]').attr('content') || '').trim();
    const imgSrcs = [];
    const saveSrc = this.generateImgName(imgSrc);
    this.downloadImg(imgSrc, saveSrc);
    // find the left imgs
    const otherImgs = await this.loadOtherImgs($, link);
    imgSrcs.push(saveSrc, ...otherImgs);
    return {
      title,
      type: 1,
      keywords,
      description,
      imgSrcs: imgSrcs,
      content
    };
  }
  async loadOtherImgs($, link) {
    const lis = $('.articleV4Page').find('>li');
    if (lis.length > 5) { // has other imgs
      const promises = [];
      for (let i = 3, len = lis.length - 2; i < len; i++) {
        const href = $(lis[i]).find('a').attr('href');
        if (think.isEmpty(href)) {
          continue;
        }
        promises.push(this.getOtherImgSrc(this.parseUrl(link, href)));
      }
      return Promise.all(promises);
    }
    return [];
  }
  getHost(url) {
    const pos = url.indexOf('://');
    const lpos = url.indexOf('/', pos + 4);
    if (lpos === -1) {
      return url;
    }
    return url.substr(0, lpos);
  }
  parseUrl(currentPage, link) {
    if (link.indexOf('://') !== -1) {
      return link;
    }
    if (link[0] === '/') { // base on the host
      return this.getHost(currentPage) + link;
    }
    return this.dirname(currentPage) + '/' + link;
  }
  async getOtherImgSrc(link) {
    const content = await this.fetchLinkContent(link);
    const $ = cheerio.load(content);
    const imgSrc = $('#picBody').find('img').attr('src');
    const saveSrc = this.generateImgName(imgSrc);
    this.downloadImg(imgSrc, saveSrc);
    return saveSrc;
  }
  async loadNews($) {
    const body = $('.contentV3_body');
    const title = $('.contentV3_title>h1').text().trim();
    if (think.isEmpty(title) && await this.albumExistsByTitle(title)) {
      return;
    }
    body.find('img').each((idx, el) => {
      const jq = $(el);
      const src = jq.attr('src');
      const saveSrc = this.generateImgName(src);
      this.downloadImg(src, saveSrc);
      jq.attr('src', imgHost + saveSrc);
      jq.removeAttr('oldsrc');
      jq.removeClass('content-picture');
    });
    body.find('a').each((idx, el) => {
      const jq = $(el);
      jq.attr('href', 'javascript:void(0)');
    });
    body.find('p').each((idx, el) => {
      const jq = $(el);
      jq.removeAttr('class');
      jq.removeAttr('style');
      jq.removeAttr('align');
      if (!jq.find('img').length) {
        const c = jq.text().trim();
        if (c) {
          jq.html(c);
        } else {
          jq.remove();
        }
      }
    });
    const keywords = ($('meta[name=keywords]').attr('content') || '').trim();
    const description = ($('meta[name=description]').attr('content') || '').trim();
    const content = body.html().trim();
    return {
      keywords,
      description,
      content,
      title,
      type: 0
    };
  }
  dirname(filename) {
    const idx = filename.lastIndexOf('/');
    if (idx !== -1) {
      return filename.substr(0, idx);
    }
    return new Error('filename can not be empty');
  }
  downloadImg(src, saveSrc) {
    const destfile = imgSaveDir + saveSrc;
    mkdirp(this.dirname(destfile), (err, made) => {
      if (!err) {
        try {
          rp.get(src).pipe(fs.createWriteStream(destfile));
        } catch (e) {
          think.logger.error('img:download fail:', src, '=>', destfile);
        }
      } else {
        think.logger.error(err);
      }
    });
  }
  getExt(src) {
    const parts = src.split('.');
    if (parts.length > 1) {
      return parts[parts.length - 1];
    }
    return '';
  }
  generateImgName(src) {
    const date = moment().format('YYYY-MM-DD');
    return '/static/upload/images/' + date + '/' + think.md5(think.uuid()) + '.' + this.getExt(src);
  }
};
